# -*- coding: utf-8 -*-
import random
import urllib

import scrapy
from scrapy.selector import Selector
import re
import requests

from douban.item import Plant


class DoubanSpiderSpider(scrapy.Spider):
    name = 'iplant_spider'
    # start_urls = ['http://www.iplant.cn/frps/ashx/getnamec.ashx?t=A']
    domain = 'http://www.iplant.cn'
    def start_requests(self):  # 由此方法通过下面链接爬取页面

        # 定义爬取的链接
        keys = ["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z"]
        url = "http://www.iplant.cn/frps/ashx/getnamec.ashx?t="
        for key in keys:
            yield scrapy.Request(url=url+key, callback=self.getPlantUrl)  # 爬取到的页面如何处理？提交给getPlantUrl方法处理
        # yield scrapy.Request(url="http://www.iplant.cn/info/Heracleum%20apaense?t=z", callback=self.getPlantData)
    def getPlantUrl(self, response):
        print("response\n")
        hrefs = response.xpath('//div//a[@href]/@href').extract()
        for href in hrefs:
            yield scrapy.Request(url=self.domain+href, callback=self.getPlantData)  # 爬取到的页面如何处理？提交给parse方法处理

    def getPlantData(self,response):
        # 参数获取
        spid = re.findall(u'var spno = "(\d+)"',response.xpath('//script[not(@src)]').extract_first())
        if len(spid)>0:
            spid = spid[0]
        else:
            spid = -1
        name = re.findall(u'http://www.iplant.cn/info/(.+)\?t=z',response.request.url)
        if len(name)>0:
            name = urllib.parse.unquote(name[0])
        else:
            name = ""
        # 获取植物介绍
        descPayload = {'key': name,'m': self._genRandomNum()}
        descJson = requests.get('http://www.iplant.cn/ashx/getfrps.ashx',descPayload).json()
        # 获取层级结构
        relPayload = {'spno':descJson['frpsspno'],'spclassid':descJson['frpsspclassid'],'t':self._genRandomNum()}
        relJson = requests.post("http://www.iplant.cn/ashx/getfrpsclass.ashx",relPayload).json()
        plant = Plant()
        plant['c_name'] = descJson['frpscname']
        plant['e_name'] = name
        plant['desc'] = Selector(text=descJson['frpsdesc']).xpath(".//p//text()").extract()
        plant['level'] = Selector(text=relJson['frpsclasstxt']).xpath(".//div//text()").extract()
        yield plant
    # 生成随机小数
    def _genRandomNum(self,len=15):
        return round(random.uniform(0,1),len)